/*
 * Copyright (c) 2016. For Intelligent Group.
 */

package com.intelligent.ispc.core.service.impl;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.intelligent.ispc.cache.SpcDataCacheFactory;
import com.intelligent.ispc.common.mapper.JsonMapper;
import com.intelligent.ispc.common.utils.*;
import com.intelligent.ispc.constant.AppConstant;
import com.intelligent.ispc.constant.ProcessConstant;
import com.intelligent.ispc.core.dto.*;
import com.intelligent.ispc.core.service.ReportService;
import com.intelligent.ispc.core.service.SysConfigService;
import com.intelligent.ispc.core.service.SystemService;
import com.intelligent.ispc.export.yield.YieldExcelBuilder;
import com.intelligent.ispc.export.yield.YieldExportWorker;
import com.intelligent.ispc.utils.*;
import com.intelligent.ispc.utils.executor.ExportThreadManager;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.io.File;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
import java.util.regex.Pattern;

import com.intelligent.ispc.core.dto.YieldParamDto;


/**
 * Created by Julia.Zhou on 2016/08/16.
 */
@Service
public class ReportServiceImpl implements ReportService {

    private final Logger logger = LoggerFactory.getLogger(ReportServiceImpl.class);

    @Autowired
    private SystemService systemService;

    @Autowired
    private SpcDataCacheFactory spcDataCacheFactory;

    @Autowired
    private SystemConfig systemConfig;

    @Autowired
    private SysConfigService sysConfigService;

    private JsonMapper jsonMapper = JsonMapper.nonDefaultMapper();


    @Override
    public FieldDataSourceDto findFieldDataSource(List<String> filePaths) {
        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.YIELD_DECODE_FILE);
        if (filePaths == null || filePaths.isEmpty()) {
            throw new ApplicationException(ExceptionMessages.ERR_13001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_YIELD_PARAMETER_INVALID));
        }
        processHelper.postProcess(10, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING));
        SysConfigDto sysConfigDto = sysConfigService.findByName(AppConstant.SYSTEM_CONFIG_SCRIPT_PATH);
        String scriptPath = null;
        if (sysConfigDto != null && StringUtils.isNotBlank(sysConfigDto.getValue())) {
            scriptPath = sysConfigDto.getValue();
        }
        if (StringUtils.isBlank(scriptPath)) {
            scriptPath = ApplicationPathUtil.getPath(AppConstant.PATH_SCRIPT, AppConstant.PATH_SCRIPT_FILENAME);
        }
        sysConfigDto = null;
        SystemConfigDto systemConfigDto = systemService.findSystemConfig();
        processHelper.postProcess(25, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_DONE));

        processHelper.postProcess(28, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_CONDITION));
        PythonUtil util = new PythonUtil();
        List<String> searchKeys = util.getSearchKeys(filePaths, scriptPath);

        List<String> attributeKeys = util.getAttributeKeys(filePaths, scriptPath);
        processHelper.postProcess(55, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_CONDITION_DONE));

        Set<String> searchKeyNames = new LinkedHashSet<>();
        FieldDataSourceDto fieldDataSourceDto = new FieldDataSourceDto();
        fieldDataSourceDto.setSearchFields(this.getSearchKeys(searchKeys, searchKeyNames, systemConfigDto));

        Set<String> searchAndAttributeKeys = new LinkedHashSet<>();
        searchAndAttributeKeys.addAll(searchKeyNames);
        searchAndAttributeKeys.addAll(this.getAttributeKeys(attributeKeys, systemConfigDto));

        processHelper.postProcess(60, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM));

        List<String> itemNames = util.getColumnKeys(filePaths, scriptPath);
        List<String> searchAndAttributeKeysResult = new LinkedList<>(searchAndAttributeKeys);
        fieldDataSourceDto.setSearchAndAttributeFields(searchAndAttributeKeysResult);

        fieldDataSourceDto.setTestItemNames(this.getTestItemKeys(itemNames));

        String errorKeys = checkSearchKeys(util, filePaths, searchAndAttributeKeysResult);
        if (StringUtils.isNotBlank(errorKeys)) {
            String[] args = new String[]{errorKeys};
            throw new ApplicationException(ExceptionMessages.ERR_13002, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_YIELD_COLUMN_INVALID, args));
        }

        processHelper.postProcess(90, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM_DONE));
        systemConfigDto = null;
        searchKeys = null;
        attributeKeys = null;
        searchKeyNames = null;
        itemNames = null;
        searchAndAttributeKeysResult = null;
        searchAndAttributeKeys = null;
        processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_FINISH));
        return fieldDataSourceDto;
    }

    @Override
    public List<YieldDto> getYield(List<String> filePath, List<DynSearchFieldDto> searchKeys, List<String> groupKeys, List<String> testItemNames, YieldItemParamDto yieldItemParamDto) {
        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.YIELD_EXECUTE_TASK);
        processHelper.postProcess(5, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_CONDITION));
        SystemConfigDto systemConfigDto = systemService.findSystemConfig();
        if (systemConfigDto == null || StringUtils.isBlank(systemConfigDto.getPrimaryKey())
                || StringUtils.isBlank(systemConfigDto.getYieldValidateKey()) || StringUtils.isBlank(systemConfigDto.getYieldValidatePassValue())) {
            logger.error("primaryKey or validate pass key is empty.");
            throw new ApplicationException(ExceptionMessages.ERR_13003, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_YIELD_CONFIG_INVALID));
        }
        String scriptPath = this.getScriptPath();
        processHelper.postProcess(15, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_CONDITION_DONE));
        processHelper.postProcess(20, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING));
        Map<String, List<Map<String, String>>> groupMap = filterGroupDatas(filePath, scriptPath, groupKeys, systemConfigDto, testItemNames, yieldItemParamDto);
        processHelper.postProcess(55, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_DONE));

        processHelper.postProcess(60, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM));
        List<YieldDto> result = Lists.newLinkedList();
        if (groupMap != null && !groupMap.isEmpty()) {
            for (String groupKey : groupMap.keySet()) {
                result.add(analysisYield(groupKey, searchKeys, systemConfigDto, groupMap.get(groupKey), testItemNames));
            }
        } else {
            YieldDto yieldDto = new YieldDto();
            yieldDto.setFirstInspect(0);
            yieldDto.setFirstPass(0);
            yieldDto.setFirstFail(0);
            yieldDto.setReTestInspect(0);
            yieldDto.setReTestPass(0);
            yieldDto.setReTestFail(0);
            yieldDto.setTotalPass(0);
            yieldDto.setTotolInspect(0);
            yieldDto.setYieldCount();
            result.add(yieldDto);
        }
        processHelper.postProcess(95, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM_DONE));
        systemConfigDto = null;
        groupMap = null;
        processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_FINISH));
        return result;
    }

    @Override
    public YieldHistoryDto getYieldHistory(List<String> filePaths, List<DynSearchFieldDto> searchKeys, List<String> groupKeys, String groupValue, Integer type,
                                           List<String> testItemNames, YieldItemParamDto yieldItemParamDto) {
        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.YIELD_HISTORY_TASK);
        processHelper.postProcess(5, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_CONDITION));
        SystemConfigDto systemConfigDto = systemService.findSystemConfig();
        if (systemConfigDto == null || StringUtils.isBlank(systemConfigDto.getPrimaryKey())
                || StringUtils.isBlank(systemConfigDto.getYieldValidateKey()) || StringUtils.isBlank(systemConfigDto.getYieldValidatePassValue())) {
            logger.error("primaryKey or validate pass key is empty.");
            throw new ApplicationException(ExceptionMessages.ERR_13003, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_YIELD_CONFIG_INVALID));
        }
        String scriptPath = this.getScriptPath();
        Set<String> seachKeyResult = this.getSearchAndAttributeKeys(filePaths, scriptPath, systemConfigDto);
        if (testItemNames != null && testItemNames.size() != 0) {
            seachKeyResult.add(testItemNames.get(0) + " ");
        }
        processHelper.postProcess(15, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_CONDITION_DONE));
        processHelper.postProcess(20, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING));
        Map<String, List<Map<String, String>>> groupMap = this.filterGroupDatas(filePaths, scriptPath, groupKeys, systemConfigDto, testItemNames, yieldItemParamDto);
        String primaryKey = systemConfigDto.getPrimaryKey();
        String validatePassKey = null;
        if (testItemNames != null && testItemNames.size() != 0) {
            validatePassKey = testItemNames.get(0);
        } else {
            validatePassKey = systemConfigDto.getYieldValidateKey();
        }
        String validateValue1 = systemConfigDto.getYieldValidatePassValue();
        String validatePassValue = null;
        String validateFailValue = null;

        if (StringUtils.isNotBlank(validateValue1) && validateValue1.split(",") != null && validateValue1.split(",").length == 2) {
            validatePassValue = validateValue1.split(",")[0];
            validateFailValue = validateValue1.split(",")[1];
        } else {
            logger.error("Yield validate value error! validateValue:[{}]", validateValue1);
            throw new ApplicationException(ExceptionMessages.ERR_13003, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_YIELD_CONFIG_INVALID));
        }
        systemConfigDto = null;
        String primaryValue = null;
        String validateValue = null;
        YieldHistoryDto yieldHistoryDto = new YieldHistoryDto();
        yieldHistoryDto.setColumnNames((String[]) seachKeyResult.toArray(new String[seachKeyResult.size()]));
        List<String[]> columValuesResult = Lists.newLinkedList();
        processHelper.postProcess(55, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_DONE));

        processHelper.postProcess(60, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM));
        if (groupMap != null && !groupMap.isEmpty()) {
            List<String[]> columnValues = Lists.newLinkedList();
            List<String> datas = Lists.newLinkedList();
            for (String groupKey : groupMap.keySet()) {
                if (groupValue == null && groupKey != null) {
                    continue;
                }
                if (groupValue == null || groupValue.equals(groupKey)) {
                    Map<String, Map<String, String>> firstAllPrimaryMap = new LinkedHashMap<>();
                    Map<String, Map<String, List<String>>> filterAllPrimaryMap = new LinkedHashMap<>();
                    Map<String, List<Map<String, String>>> filterAllPrimaryDataMap = new LinkedHashMap<>();

                    for (int i = 0; i < groupMap.get(groupKey).size(); i++) {
                        Map<String, String> row = groupMap.get(groupKey).get(i);
                        primaryValue = row.get(primaryKey);
                        validateValue = row.get(validatePassKey);
                        if (StringUtils.isBlank(primaryValue) || StringUtils.isBlank(validateValue)) {
                            continue;
                        }
                        boolean isExist = false;
                        if (!firstAllPrimaryMap.containsKey(primaryValue)) {
                            Map<String, String> resultSub = new LinkedHashMap<>();
                            resultSub.put("0", validateValue);
                            firstAllPrimaryMap.put(primaryValue, resultSub);
                        } else {
                            isExist = true;
                        }

                        //filter
                        if (searchKeys != null && !searchKeys.isEmpty()) {
                            SpcFilterUtil spcFilterUtil = new SpcFilterUtil();
                            if (!spcFilterUtil.filterData(searchKeys, row)) {
                                continue;
                            }
                        }

                        if (!filterAllPrimaryMap.containsKey(primaryValue)) {
                            Map<String, List<String>> resultSub = new LinkedHashMap<>();
                            List<String> validateValues = Lists.newLinkedList();
                            validateValues.add(validateValue);
                            if (!isExist) {
                                resultSub.put("0", validateValues);
                            } else {
                                resultSub.put("1", validateValues);
                            }
                            filterAllPrimaryMap.put(primaryValue, resultSub);
                            List<Map<String, String>> data = Lists.newLinkedList();
                            data.add(row);
                            filterAllPrimaryDataMap.put(primaryValue, data);
                            resultSub = null;
                            data = null;
                        } else {
                            if (filterAllPrimaryMap.get(primaryValue).get("1") == null) {
                                List<String> validateValues = Lists.newLinkedList();
                                validateValues.add(validateValue);
                                filterAllPrimaryMap.get(primaryValue).put("1", validateValues);
                                validateValues = null;
                            } else {
                                filterAllPrimaryMap.get(primaryValue).get("1").add(validateValue);
                            }
                            filterAllPrimaryDataMap.get(primaryValue).add(row);
                        }
                        row = null;
                    }


                    for (String primaryValueKey : filterAllPrimaryMap.keySet()) {
                        Map<String, List<String>> result = filterAllPrimaryMap.get(primaryValueKey);
                        List<String> firstPrimary = result.get("0");
                        switch (type) {
                            case 0:
                                if (firstPrimary != null && firstPrimary.size() == 1) {
                                    columnValues.clear();
                                    columnValues = this.getFirstTestColumnValues(yieldHistoryDto, primaryValueKey, filterAllPrimaryDataMap);
                                    if (columnValues != null) {
                                        columValuesResult.addAll(columnValues);
                                    }
                                }
                                break;
                            case 1:
                                if (firstPrimary != null && firstPrimary.size() == 1 && validatePassValue.equals(firstPrimary.get(0))) {
                                    columnValues.clear();
                                    columnValues = this.getFirstTestColumnValues(yieldHistoryDto, primaryValueKey, filterAllPrimaryDataMap);
                                    if (columnValues != null) {
                                        columValuesResult.addAll(columnValues);
                                    }
                                }
                                break;
                            case 2:
                                if (firstPrimary != null && firstPrimary.size() == 1 && validateFailValue.equals(firstPrimary.get(0))) {
                                    columnValues.clear();
                                    columnValues = this.getFirstTestColumnValues(yieldHistoryDto, primaryValueKey, filterAllPrimaryDataMap);
                                    if (columnValues != null) {
                                        columValuesResult.addAll(columnValues);
                                    }
                                }
                                break;
                            case 3:
                                if (firstPrimary != null && firstPrimary.size() == 1) {
                                    if (result.get("1") != null && !result.get("1").isEmpty()) {
                                        columnValues.clear();
                                        for (int i = 1; i < filterAllPrimaryDataMap.get(primaryValueKey).size(); i++) {
                                            if (datas != null && !datas.isEmpty()) {
                                                datas.clear();
                                            }
                                            for (String clomName : yieldHistoryDto.getColumnNames()) {
                                                datas.add(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(clomName));
                                            }

                                            if (datas != null && !datas.isEmpty()) {
                                                columnValues.add((String[]) datas.toArray(new String[yieldHistoryDto.getColumnNames().length]));
                                            }
                                        }
                                        columValuesResult.addAll(columnValues);

                                    }
                                } else if ((firstPrimary == null || firstPrimary.isEmpty()) && firstAllPrimaryMap.get(primaryValueKey) != null) {
                                    columnValues.clear();
                                    for (int i = 0; i < filterAllPrimaryDataMap.get(primaryValueKey).size(); i++) {
                                        if (datas != null && !datas.isEmpty()) {
                                            datas.clear();
                                        }
                                        for (String clomName : yieldHistoryDto.getColumnNames()) {
                                            datas.add(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(clomName));
                                        }
                                        if (datas != null && !datas.isEmpty()) {
                                            columnValues.add((String[]) datas.toArray(new String[yieldHistoryDto.getColumnNames().length]));
                                        }
                                    }
                                    columValuesResult.addAll(columnValues);
                                }
                                break;
                            case 4:
                                if (firstPrimary != null && firstPrimary.size() == 1) {
                                    List<String> results = result.get("1");
                                    if (results != null && !result.isEmpty()) {
                                        columnValues.clear();
                                        for (int i = 1; i < filterAllPrimaryDataMap.get(primaryValueKey).size(); i++) {
                                            if (datas != null && !datas.isEmpty()) {
                                                datas.clear();
                                            }
                                            for (String clomName : yieldHistoryDto.getColumnNames()) {
                                                if (validatePassValue.equals(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(validatePassKey))) {
                                                    datas.add(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(clomName));
                                                }
                                            }
                                            if (datas != null && !datas.isEmpty()) {
                                                columnValues.add((String[]) datas.toArray(new String[yieldHistoryDto.getColumnNames().length]));
                                            }
                                        }
                                        columValuesResult.addAll(columnValues);
                                    }
                                } else if (firstPrimary == null || firstPrimary.isEmpty()) {
                                    List<String> results = result.get("1");
                                    if (results != null && !result.isEmpty()) {
                                        columnValues.clear();
                                        for (int i = 0; i < filterAllPrimaryDataMap.get(primaryValueKey).size(); i++) {
                                            if (datas != null && !datas.isEmpty()) {
                                                datas.clear();
                                            }
                                            for (String clomName : yieldHistoryDto.getColumnNames()) {
                                                if (validatePassValue.equals(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(validatePassKey))) {
                                                    datas.add(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(clomName));
                                                }
                                            }
                                            if (datas != null && !datas.isEmpty()) {
                                                columnValues.add((String[]) datas.toArray(new String[yieldHistoryDto.getColumnNames().length]));
                                            }
                                        }
                                        columValuesResult.addAll(columnValues);
                                    }
                                }
                                break;
                            case 5:
                                if (firstPrimary != null && firstPrimary.size() == 1) {
                                    List<String> results = result.get("1");
                                    if (results != null && !result.isEmpty()) {
                                        columnValues.clear();
                                        for (int i = 1; i < filterAllPrimaryDataMap.get(primaryValueKey).size(); i++) {
                                            if (datas != null && !datas.isEmpty()) {
                                                datas.clear();
                                            }
                                            for (String clomName : yieldHistoryDto.getColumnNames()) {
                                                if (validateFailValue.equals(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(validatePassKey))) {
                                                    datas.add(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(clomName));
                                                }
                                            }

                                            if (datas != null && !datas.isEmpty()) {
                                                columnValues.add((String[]) datas.toArray(new String[yieldHistoryDto.getColumnNames().length]));
                                            }
                                        }
                                        columValuesResult.addAll(columnValues);
                                    }
                                } else if (firstPrimary == null || firstPrimary.isEmpty()) {
                                    List<String> results = result.get("1");
                                    if (results != null && !result.isEmpty()) {
                                        columnValues.clear();
                                        for (int i = 0; i < filterAllPrimaryDataMap.get(primaryValueKey).size(); i++) {
                                            if (datas != null && !datas.isEmpty()) {
                                                datas.clear();
                                            }
                                            for (String clomName : yieldHistoryDto.getColumnNames()) {
                                                if (validateFailValue.equals(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(validatePassKey))) {
                                                    datas.add(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(clomName));
                                                }
                                            }
                                            if (datas != null && !datas.isEmpty()) {
                                                columnValues.add((String[]) datas.toArray(new String[yieldHistoryDto.getColumnNames().length]));
                                            }
                                        }
                                        columValuesResult.addAll(columnValues);
                                    }
                                }
                                break;
                            default:
                                break;
                        }
                        result = null;
                    }
                    filterAllPrimaryDataMap = null;
                    filterAllPrimaryMap = null;
                    firstAllPrimaryMap = null;
                }
            }
            datas = null;
            columnValues = null;
        }
        yieldHistoryDto.setColumnValues(columValuesResult);
        processHelper.postProcess(95, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM_DONE));
        groupMap = null;
        columValuesResult = null;

        processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_FINISH));
        return yieldHistoryDto;
    }


    public YieldItemParamDto getYieldItemParam(List<String> filePaths, List<String> testItemNames) {
        Double upperLimited;
        Double lowerLimited;
        String scriptPath = this.getScriptPath();
        PythonUtil util = new PythonUtil();
        List<String> paramKeys = util.getParamKeys(filePaths, scriptPath);
        Map<String, Object> paramMap = new LinkedHashMap<>();
        for (int i = 0; i < paramKeys.size(); i++) {
            paramMap = null;
            if (paramKeys != null && !paramKeys.isEmpty()) {
                paramMap = jsonMapper.fromJson(paramKeys.get(i).toString(), Map.class);
            }
            if (paramMap.containsKey(testItemNames.get(0))) {
                break;
            }
        }

        String[] param = paramMap.get(testItemNames.get(0)).toString().replace(",", " , ").split(",");
        upperLimited = StringUtils.isNumeric(param[0].trim()) ? Double.parseDouble(param[0].trim()) : null;
        lowerLimited = StringUtils.isNumeric(param[1].trim()) ? Double.parseDouble(param[1].trim()) : null;
        YieldItemParamDto yieldItemParamDto = new YieldItemParamDto();
        yieldItemParamDto.setUsl(upperLimited);
        yieldItemParamDto.setLsl(lowerLimited);
        return yieldItemParamDto;
    }

    @Override
    public YieldExportDto getSummaryAndDetail(List<String> filePaths, List<DynSearchFieldDto> searchKeys, List<String> groupKeys, List<String> testItemNames, YieldItemParamDto yieldItemParamDto) {
        SystemConfigDto systemConfigDto = systemService.findSystemConfig();

        if (systemConfigDto == null || StringUtils.isBlank(systemConfigDto.getPrimaryKey())
                || StringUtils.isBlank(systemConfigDto.getYieldValidateKey()) || StringUtils.isBlank(systemConfigDto.getYieldValidatePassValue())) {
            logger.error("primaryKey or validate pass key is empty.");
            throw new ApplicationException(ExceptionMessages.ERR_13004, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_YIELD_EXPORT_PARAMETER_INVALID
            ));
        }
        YieldExportDto yieldExportDto = new YieldExportDto();
        Map<String, List<YieldHistoryDto>> yieldHistoryDtoMap = new LinkedHashMap<>();

        YieldHistoryDto yieldHistoryDto = new YieldHistoryDto();

        String scriptPath = this.getScriptPath();
        Set<String> seachKeyResult = this.getSearchAndAttributeKeys(filePaths, scriptPath, systemConfigDto);
        if (testItemNames != null && testItemNames.size() != 0) {
            seachKeyResult.add(testItemNames.get(0) + " ");
            //seachKeyResult.add(testItemNames.get(0));
        }
        Map<String, List<Map<String, String>>> groupMap = this.filterGroupDatas(filePaths, scriptPath, groupKeys, systemConfigDto, testItemNames, null);
        String primaryKey = systemConfigDto.getPrimaryKey();
        String validatePassKey = null;
        if (testItemNames != null && testItemNames.size() != 0) {
            validatePassKey = testItemNames.get(0);
        } else {
            validatePassKey = systemConfigDto.getYieldValidateKey();
        }
        String validateValue1 = systemConfigDto.getYieldValidatePassValue();
        String validatePassValue = null;
        String validateFailValue = null;

        if (StringUtils.isNotBlank(validateValue1) && validateValue1.split(",") != null && validateValue1.split(",").length == 2) {
            validatePassValue = validateValue1.split(",")[0];
            validateFailValue = validateValue1.split(",")[1];
        } else {
            logger.error("Yield validate value error! validateValue:[{}]", validateValue1);
            throw new ApplicationException(ExceptionMessages.ERR_13004, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_YIELD_EXPORT_PARAMETER_INVALID));
        }

        String primaryValue = null;
        String validateValue = null;
        yieldHistoryDto.setColumnNames((String[]) seachKeyResult.toArray(new String[seachKeyResult.size()]));

        if (groupMap != null && !groupMap.isEmpty()) {
            for (String groupKey : groupMap.keySet()) {
                List<YieldHistoryDto> yieldHistoryDtos = Lists.newArrayList();
                List<String[]> firstInspectResult = Lists.newLinkedList();
                List<String[]> firstPassResult = Lists.newLinkedList();
                List<String[]> firstFailResult = Lists.newLinkedList();
                List<String[]> retestInspectResult = Lists.newLinkedList();
                List<String[]> retestPassResult = Lists.newLinkedList();
                List<String[]> retestFailResult = Lists.newLinkedList();
                Map<String, Map<String, String>> firstAllPrimaryMap = new LinkedHashMap<>();
                Map<String, Map<String, List<String>>> filterAllPrimaryMap = new LinkedHashMap<>();
                Map<String, List<Map<String, String>>> filterAllPrimaryDataMap = new LinkedHashMap<>();
                for (int i = 0; i < groupMap.get(groupKey).size(); i++) {
                    Map<String, String> row = groupMap.get(groupKey).get(i);
                    primaryValue = row.get(primaryKey);
                    validateValue = row.get(validatePassKey);
                    if (StringUtils.isBlank(primaryValue) || StringUtils.isBlank(validateValue)) {
                        continue;
                    }
                    boolean isExist = false;
                    if (!firstAllPrimaryMap.containsKey(primaryValue)) {
                        Map<String, String> resultSub = new LinkedHashMap<>();
                        resultSub.put("0", validateValue);
                        firstAllPrimaryMap.put(primaryValue, resultSub);
                        resultSub = null;
                    } else {
                        isExist = true;
                    }

                    //filter
                    if (searchKeys != null && !searchKeys.isEmpty()) {
                        SpcFilterUtil spcFilterUtil = new SpcFilterUtil();
                        if (!spcFilterUtil.filterData(searchKeys, row)) {
                            continue;
                        }
                    }

                    if (!filterAllPrimaryMap.containsKey(primaryValue)) {
                        Map<String, List<String>> resultSub = new LinkedHashMap<>();
                        List<String> validateValues = Lists.newLinkedList();
                        validateValues.add(validateValue);
                        if (!isExist) {
                            resultSub.put("0", validateValues);
                        } else {
                            resultSub.put("1", validateValues);
                        }
                        filterAllPrimaryMap.put(primaryValue, resultSub);
                        List<Map<String, String>> data = Lists.newLinkedList();
                        data.add(row);
                        filterAllPrimaryDataMap.put(primaryValue, data);
                        data = null;
                        resultSub = null;
                        validateValues = null;
                    } else {
                        if (filterAllPrimaryMap.get(primaryValue).get("1") == null) {
                            List<String> validateValues = Lists.newLinkedList();
                            validateValues.add(validateValue);
                            filterAllPrimaryMap.get(primaryValue).put("1", validateValues);
                            validateValues = null;
                        } else {
                            filterAllPrimaryMap.get(primaryValue).get("1").add(validateValue);
                        }
                        filterAllPrimaryDataMap.get(primaryValue).add(row);
                    }

                }

                List<String[]> columnValues = null;
                for (String primaryValueKey : filterAllPrimaryMap.keySet()) {
                    Map<String, List<String>> result = filterAllPrimaryMap.get(primaryValueKey);
                    List<String> firstPrimary = result.get("0");
                    //First
                    if (firstPrimary != null && firstPrimary.size() == 1) {
                        columnValues = this.getFirstTestColumnValues(yieldHistoryDto, primaryValueKey, filterAllPrimaryDataMap);
                        //First Inspect
                        if (columnValues != null) {
                            firstInspectResult.addAll(columnValues);
                        }

                        //First Pass
                        if (validatePassValue.equals(firstPrimary.get(0))) {
                            columnValues = this.getFirstTestColumnValues(yieldHistoryDto, primaryValueKey, filterAllPrimaryDataMap);
                            if (columnValues != null) {
                                firstPassResult.addAll(columnValues);
                            }
                        }

                        //First Fail
                        if (validateFailValue.equals(firstPrimary.get(0))) {
                            columnValues = this.getFirstTestColumnValues(yieldHistoryDto, primaryValueKey, filterAllPrimaryDataMap);
                            if (columnValues != null) {
                                firstFailResult.addAll(columnValues);
                            }
                        }
                        columnValues = null;
                    }

                    //Retest
                    if (firstPrimary != null && firstPrimary.size() == 1) {
                        if (result.get("1") != null && !result.get("1").isEmpty()) {
                            for (int i = 1; i < filterAllPrimaryDataMap.get(primaryValueKey).size(); i++) {
                                List<String> reInspectDatas = Lists.newLinkedList();
                                List<String> rePassDatas = Lists.newLinkedList();
                                List<String> reFailDatas = Lists.newLinkedList();
                                for (String clomName : yieldHistoryDto.getColumnNames()) {
                                    //Retest Inspect
                                    reInspectDatas.add(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(clomName));

                                    //Rest Pass
                                    if (validatePassValue.equals(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(validatePassKey))) {
                                        rePassDatas.add(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(clomName));
                                    }
                                    //Rest Fail
                                    if (validateFailValue.equals(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(validatePassKey))) {
                                        reFailDatas.add(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(clomName));
                                    }
                                }

                                if (reInspectDatas != null && !reInspectDatas.isEmpty()) {
                                    retestInspectResult.add((String[]) reInspectDatas.toArray(new String[yieldHistoryDto.getColumnNames().length]));
                                }

                                if (rePassDatas != null && !rePassDatas.isEmpty()) {
                                    retestPassResult.add((String[]) rePassDatas.toArray(new String[yieldHistoryDto.getColumnNames().length]));
                                }

                                if (reFailDatas != null && !reFailDatas.isEmpty()) {
                                    retestFailResult.add((String[]) reFailDatas.toArray(new String[yieldHistoryDto.getColumnNames().length]));
                                }
                                reInspectDatas = null;
                                rePassDatas = null;
                                reFailDatas = null;
                            }
                        }
                    } else if ((firstPrimary == null || firstPrimary.isEmpty()) && firstAllPrimaryMap.get(primaryValueKey) != null) {
                        for (int i = 0; i < filterAllPrimaryDataMap.get(primaryValueKey).size(); i++) {
                            List<String> reInspectDatas = Lists.newLinkedList();
                            List<String> rePassDatas = Lists.newLinkedList();
                            List<String> reFailDatas = Lists.newLinkedList();
                            for (String clomName : yieldHistoryDto.getColumnNames()) {
                                //Retest Inspect
                                reInspectDatas.add(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(clomName));

                                //Rest Pass
                                if (validatePassValue.equals(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(validatePassKey))) {
                                    rePassDatas.add(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(clomName));
                                }
                                //Rest Fail
                                if (validateFailValue.equals(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(validatePassKey))) {
                                    reFailDatas.add(filterAllPrimaryDataMap.get(primaryValueKey).get(i).get(clomName));
                                }
                            }
                            if (reInspectDatas != null && !reInspectDatas.isEmpty()) {
                                retestInspectResult.add((String[]) reInspectDatas.toArray(new String[yieldHistoryDto.getColumnNames().length]));
                            }

                            if (rePassDatas != null && !rePassDatas.isEmpty()) {
                                retestPassResult.add((String[]) rePassDatas.toArray(new String[yieldHistoryDto.getColumnNames().length]));
                            }

                            if (reFailDatas != null && !reFailDatas.isEmpty()) {
                                retestFailResult.add((String[]) reFailDatas.toArray(new String[yieldHistoryDto.getColumnNames().length]));
                            }
                        }
                    }
                    result = null;
                }
                YieldHistoryDto firstInspect = new YieldHistoryDto();
                firstInspect.setColumnNames(yieldHistoryDto.getColumnNames());
                firstInspect.setColumnValues(firstInspectResult);
                yieldHistoryDtos.add(firstInspect);
                firstInspect = null;
                firstInspectResult = null;

                YieldHistoryDto firstPass = new YieldHistoryDto();
                firstPass.setColumnNames(yieldHistoryDto.getColumnNames());
                firstPass.setColumnValues(firstPassResult);
                yieldHistoryDtos.add(firstPass);
                firstPass = null;
                firstPassResult = null;

                YieldHistoryDto firstFail = new YieldHistoryDto();
                firstFail.setColumnNames(yieldHistoryDto.getColumnNames());
                firstFail.setColumnValues(firstFailResult);
                yieldHistoryDtos.add(firstFail);
                firstFail = null;
                firstFailResult = null;

                YieldHistoryDto reTestInspect = new YieldHistoryDto();
                reTestInspect.setColumnNames(yieldHistoryDto.getColumnNames());
                reTestInspect.setColumnValues(retestInspectResult);
                yieldHistoryDtos.add(reTestInspect);
                reTestInspect = null;
                retestInspectResult = null;

                YieldHistoryDto rePassInspect = new YieldHistoryDto();
                rePassInspect.setColumnNames(yieldHistoryDto.getColumnNames());
                rePassInspect.setColumnValues(retestPassResult);
                yieldHistoryDtos.add(rePassInspect);
                rePassInspect = null;
                retestPassResult = null;

                YieldHistoryDto reFailInspect = new YieldHistoryDto();
                reFailInspect.setColumnNames(yieldHistoryDto.getColumnNames());
                reFailInspect.setColumnValues(retestFailResult);
                yieldHistoryDtos.add(reFailInspect);
                yieldHistoryDtoMap.put(groupKey, yieldHistoryDtos);
                reFailInspect = null;
                retestFailResult = null;
            }
        }
        yieldExportDto.setYieldDtos(this.getYield(filePaths, searchKeys, groupKeys, testItemNames, null));
        yieldExportDto.setYieldHistoryDtoList(yieldHistoryDtoMap);
        yieldHistoryDtoMap = null;
        return yieldExportDto;
    }


    @Override
    public String exportSummary(List<String> filePath, List<DynSearchFieldDto> searchKeys, List<String> groupKeys, List<String> testItemNames, YieldItemParamDto yieldItemParamDto) {
        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.YIELD_EXPORT_FILE);
        processHelper.postProcess(5, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE));

        if (filePath == null || filePath.isEmpty()) {
            throw new ApplicationException(ExceptionMessages.ERR_13004, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_YIELD_EXPORT_PARAMETER_INVALID));
        }
        processHelper.postProcess(10, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE_DONE));
        processHelper.postProcess(15, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_DISENV));

        //assemble the param of multithreading export
        DateTimeFormatter format = DateTimeFormat.forPattern("yyyyMMddHHmmss");
        DateTime now = new DateTime();
        String fix = now.toString(format);
        StringBuilder sb = new StringBuilder();
        sb.append("Yield_Result").append("_").append(fix).append(".xlsx");
        String fileName = sb.toString();
        String savePath = null;
        String basePath = sysConfigService.findByName(AppConstant.SYSTEM_CONFIG_FIELD_PATH).getValue();
        if (StringUtils.isNotBlank(basePath)) {
            savePath = basePath + "/yield/";
        } else {
            savePath = ApplicationPathUtil.getCanonicalPath() + "export/yield/";
        }

        File file = new File(savePath);
        if (!file.exists()) {
            file.mkdirs();
        }

        processHelper.postProcess(savePath);
        String exportFilePath = savePath + fileName;

        logger.debug("Export yield only result to file:{}", fileName);

        try {
            boolean ifSucceed = false;
            ExportThreadManager exportThreadManager = ExportThreadManager.getExportThreadManager();
            Callable<Boolean> callableTaskOne = () -> exportSummary(filePath, searchKeys, groupKeys, testItemNames, yieldItemParamDto, exportFilePath, processHelper, exportThreadManager);
            FutureTask<Boolean> futureTaskOne = exportThreadManager.initYieldExportThread("TaskOne", callableTaskOne);
            exportThreadManager.startYieldExportThread();

            try {
                ifSucceed = futureTaskOne.get();

            } catch (InterruptedException e) {
                futureTaskOne.cancel(true);
            } catch (ExecutionException e) {
                e.printStackTrace();
            }
            if (ifSucceed) {
                processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_FINISH) + exportFilePath);
            } else {
                processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_FAIL));
            }
        } catch (ApplicationException e) {
            logger.error("export yield only result failed!", e);
            throw new ApplicationException(e.getCode(), e.getMessage());
        }
        return exportFilePath;
    }


    @Override
    public String exportSummaryAndDetail(List<String> filePath, List<DynSearchFieldDto> searchKeys, List<String> groupKeys, List<String> testItemNames, YieldItemParamDto yieldItemParamDto) {
        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.YIELD_EXPORT_FILE);
        processHelper.postProcess(5, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE));

        if (filePath == null || filePath.isEmpty()) {
            throw new ApplicationException(ExceptionMessages.ERR_13004, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_YIELD_EXPORT_PARAMETER_INVALID));
        }
        processHelper.postProcess(10, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE_DONE));
        processHelper.postProcess(15, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_DISENV));

        //assemble the param of multithreading export
        DateTimeFormatter format = DateTimeFormat.forPattern("yyyyMMddHHmmss");
        DateTime now = new DateTime();
        String fix = now.toString(format);
        StringBuilder sb = new StringBuilder();
        sb.append("Yield_Result_Data").append("_").append(fix).append(".xlsx");
        String fileName = sb.toString();

        String savePath = null;
        String basePath = sysConfigService.findByName(AppConstant.SYSTEM_CONFIG_FIELD_PATH).getValue();
        if (StringUtils.isNotBlank(basePath)) {
            savePath = basePath + "/yield/";
        } else {
            savePath = ApplicationPathUtil.getCanonicalPath() + "export/yield/";
        }
        File file = new File(savePath);
        if (!file.exists()) {
            file.mkdirs();
        }

        processHelper.postProcess(savePath);
        String exportFilePath = savePath + fileName;

        logger.debug("Export yield only result to file:{}", fileName);

        try {
            boolean ifSucceed = false;
            ExportThreadManager exportThreadManager = ExportThreadManager.getExportThreadManager();
            Callable<Boolean> callableTaskOne = () -> exportSummaryAndDetail(filePath, searchKeys, groupKeys, testItemNames, yieldItemParamDto, exportFilePath, fileName, processHelper, exportThreadManager);
            FutureTask<Boolean> futureTaskOne = exportThreadManager.initYieldExportThread("TaskOne", callableTaskOne);
            exportThreadManager.startYieldExportThread();

            try {
                ifSucceed = futureTaskOne.get();

            } catch (InterruptedException e) {
                futureTaskOne.cancel(true);
            } catch (ExecutionException e) {
                e.printStackTrace();
            }
            if (ifSucceed) {
                processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_FINISH) + exportFilePath);
            } else {
                processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_FAIL));
            }
        } catch (ApplicationException e) {
            logger.error("export yield only result failed!", e);
            throw new ApplicationException(e.getCode(), e.getMessage());
        }
        return exportFilePath;
    }


    private YieldDto analysisYield(String groupKey, List<DynSearchFieldDto> searchKeys, SystemConfigDto systemConfigDto, List<Map<String, String>> testDatas, List<String> testItemNames) {
        String primaryKey = systemConfigDto.getPrimaryKey();
        String validatePassKey = null;
        if (testItemNames != null && testItemNames.size() != 0) {
            validatePassKey = testItemNames.get(0);
        } else {
            validatePassKey = systemConfigDto.getYieldValidateKey();
        }
        String validateValue1 = systemConfigDto.getYieldValidatePassValue();
        String validatePassValue = null;
        String validateFailValue = null;
        if (StringUtils.isNotBlank(validateValue1) && validateValue1.split(",") != null && validateValue1.split(",").length == 2) {
            validatePassValue = validateValue1.split(",")[0];
            validateFailValue = validateValue1.split(",")[1];
        } else {
            logger.error("Yield validate value error! validateValue:[{}]", validateValue1);
            throw new ApplicationException(ExceptionMessages.ERR_13003, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_YIELD_CONFIG_INVALID));
        }

        long firstPass = 0;
        long firstFail = 0;
        long rePass = 0;
        long reFail = 0;
        long totalPass = 0;
        long total = 0;
        String primaryValue = null;
        String validateValue = null;

        YieldDto yieldDto = new YieldDto();
        yieldDto.setGroupKey(groupKey);
        Map<String, Map<Long, String>> firstAllPrimaryMap = new LinkedHashMap<>();
        Map<String, Map<Long, String>> filterAllPrimaryMap = new LinkedHashMap<>();
        for (int i = 0; i < testDatas.size(); i++) {
            Map<String, String> row = testDatas.get(i);
            primaryValue = row.get(primaryKey);
            validateValue = row.get(validatePassKey);
            if (StringUtils.isBlank(primaryValue) || StringUtils.isBlank(validateValue)) {
                continue;
            }
            boolean isExist = false;
            if (!firstAllPrimaryMap.containsKey(primaryValue)) {
                Map<Long, String> resultSub = new LinkedHashMap<>();
                resultSub.put(0L, validateValue);
                firstAllPrimaryMap.put(primaryValue, resultSub);
                resultSub = null;
            } else {
                isExist = true;
            }

            //filter
            if (searchKeys != null && !searchKeys.isEmpty()) {
                SpcFilterUtil spcFilterUtil = new SpcFilterUtil();
                if (!spcFilterUtil.filterData(searchKeys, row)) {
                    continue;
                }
            }

            if (!filterAllPrimaryMap.containsKey(primaryValue)) {
                Map<Long, String> resultSub = new LinkedHashMap<>();
                if (!isExist) {
                    resultSub.put(0L, validateValue);
                } else {
                    resultSub.put(1L, validateValue);
                }
                filterAllPrimaryMap.put(primaryValue, resultSub);
                resultSub = null;
            } else {
                if (validatePassValue.equals(filterAllPrimaryMap.get(primaryValue).get(1L))) {
                    continue;
                }
                filterAllPrimaryMap.get(primaryValue).put(1L, validateValue);
            }
            row = null;
        }
        for (String primaryValueKey : filterAllPrimaryMap.keySet()) {
            Map<Long, String> result = filterAllPrimaryMap.get(primaryValueKey);
            String firstPrimary = result.get(0L);
            if (StringUtils.isNotBlank(firstPrimary)) {
                if (firstPrimary.equals(validatePassValue)) {
                    firstPass++;
                    totalPass++;
                } else if (firstPrimary.equals(validateFailValue)) {
                    firstFail++;

                    if (validatePassValue.equals(result.get(1L))) {
                        rePass++;
                        totalPass++;
                    } else {
                        if (validateFailValue.equals(result.get(1L))) {
                            reFail++;
                        }
                    }
                }
            } else if (StringUtils.isBlank(firstPrimary) && firstAllPrimaryMap.get(primaryValueKey) != null) {
                Map<Long, String> result1 = firstAllPrimaryMap.get(primaryValueKey);
                String firstPrimary1 = result1.get(0L);
                if (StringUtils.isNotBlank(firstPrimary1) && firstPrimary1.equals(validateFailValue)) {
                    if (result.values().contains(validatePassValue)) {
                        rePass++;
                        totalPass++;
                    } else {
                        if (result.values().contains(validateFailValue)) {
                            reFail++;
                        }
                    }
                }
                Map<Long, String> result2 = filterAllPrimaryMap.get(primaryValueKey);
                if (firstPrimary1.equals(validatePassValue) && StringUtils.isNoneBlank(result2.get(1L))) {
                    totalPass++;
                }
                result1 = null;
            }
            result = null;
        }

        total = filterAllPrimaryMap.size();
        yieldDto.setFirstPass(firstPass);
        yieldDto.setFirstFail(firstFail);
        yieldDto.setReTestPass(rePass);
        yieldDto.setReTestFail(reFail);
        yieldDto.setTotalPass(totalPass);
        yieldDto.setTotolInspect(total);
        yieldDto.setFirstInspect(yieldDto.getFirstPass() + yieldDto.getFirstFail());
        yieldDto.setReTestInspect(yieldDto.getReTestPass() + yieldDto.getReTestFail());
        yieldDto.setYieldCount();
        filterAllPrimaryMap = null;
        firstAllPrimaryMap = null;
        return yieldDto;
    }

    private List<String[]> getFirstTestColumnValues(YieldHistoryDto yieldHistoryDto, String primaryValueKey, Map<String, List<Map<String, String>>> filterAllPrimaryDataMap) {
        List<String[]> columnValues = null;
        List<String> datas = Lists.newLinkedList();

        for (String clomName : yieldHistoryDto.getColumnNames()) {
            if (filterAllPrimaryDataMap.get(primaryValueKey) != null && filterAllPrimaryDataMap.get(primaryValueKey).get(0) != null) {
                datas.add(filterAllPrimaryDataMap.get(primaryValueKey).get(0).get(clomName));
            }
        }
        if (datas != null && !datas.isEmpty()) {
            columnValues = Lists.newLinkedList();
            columnValues.add((String[]) datas.toArray(new String[yieldHistoryDto.getColumnNames().length]));
        }
        datas = null;
        return columnValues;
    }

    private Map<String, List<Map<String, String>>> filterGroupDatas(List<String> filePaths, String scriptPath, List<String> groupKeys, SystemConfigDto systemConfigDto,
                                                                    List<String> testItemNames, YieldItemParamDto yieldItemParamDto) {
        String validatePassKey = null;
        if (testItemNames != null && testItemNames.size() != 0) {
            validatePassKey = testItemNames.get(0);
        } else {
            validatePassKey = systemConfigDto.getYieldValidateKey();
        }
        String validateValue1 = systemConfigDto.getYieldValidatePassValue();
        String validatePassValue = null;
        String validateFailValue = null;
        String primaryKey = systemConfigDto.getPrimaryKey();

        if (StringUtils.isNotBlank(validateValue1) && validateValue1.split(",") != null && validateValue1.split(",").length == 2) {
            validatePassValue = validateValue1.split(",")[0];
            validateFailValue = validateValue1.split(",")[1];
        } else {
            logger.error("Yield validate value error! validateValue:[{}]", validateValue1);
            throw new ApplicationException(ExceptionMessages.ERR_13003, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_YIELD_CONFIG_INVALID));
        }

        String groupKey1 = null;
        String groupKey2 = null;
        if (groupKeys != null && !groupKeys.isEmpty()) {
            if (groupKeys.size() == 1) {
                groupKey1 = groupKeys.get(0);
            } else if (groupKeys.size() == 2) {
                groupKey1 = groupKeys.get(0);
                groupKey2 = groupKeys.get(1);
            }
        }
        Map<String, List<Map<String, String>>> groupMap = new LinkedHashMap<>();
        Map<String, List<String>> testDatas = this.parseFieldData(filePaths, scriptPath, testItemNames);
        if (testItemNames != null && testItemNames.size() != 0) {
            testDatas = getTestResult(testDatas, testItemNames, filePaths, scriptPath, validatePassValue, validateFailValue, yieldItemParamDto);
        }


        if (testDatas != null && !testDatas.isEmpty()) {
            for (String mapKey : testDatas.keySet()) {
                if (mapKey.contains(AppConstant.DATA_INFO)) {
                    List<Map> dataList = jsonMapper.fromJson(testDatas.get(mapKey).toString(), List.class);
                    if (dataList != null && !dataList.isEmpty()) {
                        String groupValue = null;
                        for (int i = 0; i < dataList.size(); i++) {
                            Map<String, String> row = dataList.get(i);
                            if (StringUtils.isBlank(row.get(primaryKey)) || StringUtils.isBlank(row.get(validatePassKey))) {
                                continue;
                            }
                            if (!validatePassValue.equals(row.get(validatePassKey)) && !validateFailValue.equals(row.get(validatePassKey))) {
//                                logger.error("Yield validate value error! validateValue:[{}]", row.get(validatePassKey));
//                                throw new ApplicationException(ExceptionMessages.EXCEPTION_YIELD_DATA_VALIDATE_RESULT_ERROR);
                                continue;
                            }

                            if (StringUtils.isBlank(row.get(systemConfigDto.getPrimaryKey()))) {
                                continue;
                            }
                            if (StringUtils.isNotBlank(groupKey1) && StringUtils.isBlank(groupKey2)) {
                                groupValue = row.get(groupKey1);
                            }
                            if (StringUtils.isBlank(groupKey1) && StringUtils.isNotBlank(groupKey2)) {
                                groupValue = row.get(groupKey2);
                            } else if (StringUtils.isNotBlank(groupKey1) && StringUtils.isNotBlank(groupKey2)) {
                                groupValue = row.get(groupKey1) + AppConstant.GLOBAL_SEPARATOR + row.get(groupKey2);
                            }
                            if (!groupMap.containsKey(groupValue)) {
                                List<Map<String, String>> dataListTemp = Lists.newLinkedList();
                                dataListTemp.add(row);
                                groupMap.put(groupValue, dataListTemp);
                                dataListTemp = null;
                            } else {
                                groupMap.get(groupValue).add(row);
                            }
                            row = null;
                        }
                    }
                    dataList = null;
                }
            }
        }
        testDatas = null;
        return groupMap;
    }

    private Map<String, List<String>> parseFieldData(List<String> filePaths, String scriptPath, List<String> columnNames) {
        PythonUtil util = new PythonUtil();
        return util.getDataByName(filePaths, columnNames, scriptPath);
    }

    private List<String> parseSearchCondition(List<String> filePaths, String scriptPath, List<DynSearchFieldDto> searchKeys, SystemConfigDto systemConfigDto) {
        return new ArrayList<String>(new SpcFilterUtil().parseSearchKey(searchKeys));
    }

    private Set<String> getSearchAndAttributeKeys(List<String> filePaths, String scriptPath, SystemConfigDto systemConfigDto) {
        PythonUtil util = new PythonUtil();
        List<String> searchSourceKeys = util.getSearchKeys(filePaths, scriptPath);

        PythonUtil util1 = new PythonUtil();
        List<String> attributeSourceKeys = util1.getAttributeKeys(filePaths, scriptPath);

        Set<String> searchKeyNames = new LinkedHashSet<>();
        FieldDataSourceDto fieldDataSourceDto = new FieldDataSourceDto();
        fieldDataSourceDto.setSearchFields(this.getSearchKeys(searchSourceKeys, searchKeyNames, systemConfigDto));

        Set<String> result = new LinkedHashSet<>();
        result.addAll(searchKeyNames);
        result.addAll(this.getAttributeKeys(attributeSourceKeys, systemConfigDto));
        searchSourceKeys = null;
        attributeSourceKeys = null;
        searchKeyNames = null;
        return result;
    }

    private List<DynSearchFieldDto> getSearchKeys(List<String> searchKeysObject, Set<String> searchKeyNames, SystemConfigDto systemConfigDto) {
        List<List<String>> searchKeysList;
        Set<String> searchKeys;
        if (searchKeysObject == null) {
            searchKeys = new LinkedHashSet<>(systemConfigDto.getSearchKeyKeys());
            searchKeyNames.addAll(searchKeys);
        } else {
            searchKeys = new LinkedHashSet<>();
            searchKeysList = jsonMapper.fromJson(searchKeysObject.toString(), List.class);
            searchKeysList.forEach(searchKeys::addAll);
            searchKeyNames.addAll(searchKeys);
        }
        List<String> allSearchKeys = Lists.newArrayList(searchKeys);

        List<DynSearchFieldDto> dynSearchFieldDtos = new ArrayList<>();
        allSearchKeys.forEach((searchKey) -> {
                    DynSearchFieldDto dynSearchFieldDto = new DynSearchFieldDto();
                    dynSearchFieldDto.setName(searchKey);
                    dynSearchFieldDtos.add(dynSearchFieldDto);
                    dynSearchFieldDto = null;
                }
        );
        searchKeysList = null;
        allSearchKeys = null;
        searchKeys = null;
        return dynSearchFieldDtos;
    }

    private Set<String> getAttributeKeys(List<String> attributeKeysObject, SystemConfigDto systemConfigDto) {
        Set<String> attributeKeys;
        if (attributeKeysObject == null) {
            attributeKeys = new LinkedHashSet<>(systemConfigDto.getSearchKeyKeys());
        } else {
            attributeKeys = new LinkedHashSet<>();
            List<List<String>> attributeKeysList;
            attributeKeysList = jsonMapper.fromJson(attributeKeysObject.toString(), List.class);
            attributeKeysList.forEach(attributeKeys::addAll);
            attributeKeysList = null;
        }

        return attributeKeys;

    }

    private String getScriptPath() {
        SysConfigDto sysConfigDto = sysConfigService.findByName(AppConstant.SYSTEM_CONFIG_SCRIPT_PATH);
        String scriptPath = null;
        if (sysConfigDto != null && StringUtils.isNotBlank(sysConfigDto.getValue())) {
            scriptPath = sysConfigDto.getValue();
        }
        if (StringUtils.isBlank(scriptPath)) {
            scriptPath = ApplicationPathUtil.getPath(AppConstant.PATH_SCRIPT, AppConstant.PATH_SCRIPT_FILENAME);
        }
        return scriptPath;
    }

    private List<String> getTestItemKeys(List<String> itemNameObject) {
        List<List<String>> itemNameList = jsonMapper.fromJson(itemNameObject.toString(), List.class);
        Set<String> testItemNamesSet = new LinkedHashSet<>();
        itemNameList.forEach(testItemNamesSet::addAll);
        List<String> testItemNames = new ArrayList<>(testItemNamesSet);
        itemNameList = null;
        return testItemNames;

    }

    /*
    * If the user specifies a test item, then use the test data and usl, lsl to calculate the Yield.
    * */
    private Map<String, List<String>> getTestResult(Map<String, List<String>> testDatas, List<String> testItemNames, List<String> filePaths, String scriptPath,
                                                    String validatePassValue, String validateFailValue, YieldItemParamDto yieldItemParamDto) {
        Double upperLimited = null;
        Double lowerLimited = null;
        String scientificNotationRegx = "^([+-]?[1-9].[0-9]+[Ee][+-]?[0-9]+)$";
        Pattern pattern = Pattern.compile(scientificNotationRegx);
        PythonUtil util = new PythonUtil();
        List<String> paramKeys = util.getParamKeys(filePaths, scriptPath);
        Map<String, Object> paramMap = Maps.newLinkedHashMap();

        if (yieldItemParamDto != null) {
            upperLimited = yieldItemParamDto.getUsl();
            lowerLimited = yieldItemParamDto.getLsl();
        } else {
            String[] param = null;
            if (paramKeys != null && !paramKeys.isEmpty()) {
                for (int i = 0; i < paramKeys.size(); i++) {
                    paramMap = jsonMapper.fromJson(paramKeys.get(i).toString(), Map.class);
                    if (paramMap.get(testItemNames.get(0)) != null && StringUtils.isNotBlank(paramMap.get(testItemNames.get(0)).toString())) {
                        param = paramMap.get(testItemNames.get(0)).toString().replace(",", " , ").split(",");
                        break;
                    }
                }
            }

            if (param != null && param.length != 0) {
                //String[] param = paramMap.get(testItemNames.get(0)).toString().replace(",", " , ").split(",");
                upperLimited = StringUtils.isNumeric(param[0].trim()) ? Double.parseDouble(param[0].trim()) : null;
                lowerLimited = StringUtils.isNumeric(param[1].trim()) ? Double.parseDouble(param[1].trim()) : null;
            } else {
                upperLimited = null;
                lowerLimited = null;
            }
        }
        if (testDatas != null && !testDatas.isEmpty()) {
            for (String mapKey : testDatas.keySet()) {
                //
                if (mapKey.contains(AppConstant.DATA_INFO)) {
                    List<String> data = Lists.newArrayList();
                    List<Map> dataList = jsonMapper.fromJson(testDatas.get(mapKey).toString(), List.class);
                    if (dataList != null && !dataList.isEmpty() && dataList.get(0).containsKey(testItemNames.get(0))) {
                        for (int i = 0; i < dataList.size(); i++) {
                            Map<String, String> row = dataList.get(i);
                            row.put(testItemNames.get(0) + " ", row.get(testItemNames.get(0)));
                            if ((upperLimited == null && lowerLimited == null) || (!StringUtils.isNumeric(row.get(testItemNames.get(0))) && !pattern.matcher(row.get(testItemNames.get(0))).matches())) {
                                row.put(testItemNames.get(0), " ");
                            } else {
                                Double testItemValue = Double.parseDouble(row.get(testItemNames.get(0)));
                                if (upperLimited != null && lowerLimited != null) {
                                    if (testItemValue <= upperLimited && testItemValue >= lowerLimited) {
                                        row.put(testItemNames.get(0), validatePassValue);
                                    } else {
                                        row.put(testItemNames.get(0), validateFailValue);
                                    }
                                } else if (upperLimited != null) {
                                    if (testItemValue <= upperLimited) {
                                        row.put(testItemNames.get(0), validatePassValue);
                                    } else {
                                        row.put(testItemNames.get(0), validateFailValue);
                                    }
                                } else if (lowerLimited != null) {
                                    if (testItemValue >= lowerLimited) {
                                        row.put(testItemNames.get(0), validatePassValue);
                                    } else {
                                        row.put(testItemNames.get(0), validateFailValue);
                                    }
                                } else {
                                    row.put(testItemNames.get(0), " ");
                                }
                            }
                            data.add(jsonMapper.toJson(dataList.get(i)));
                        }
                        testDatas.put(mapKey, data);
                    }
                }
            }
        }
        return testDatas;
    }

    /*
     * Multithreaded export Summary of Yeild
     * @param filePath          source file path
     * @param searchKeys        search condition
     * @param groupKeys         grouping
     * @param testItemNames     name of test item
     * @param yieldItemParamDto
     * @param exportFilePath    file exporting path
     * @param processHelper     progress bar
     * @param exportThreadManager  class of Multithreaded export
     * @return                  it will return true when successfully export excel
     */
    private boolean exportSummary(List<String> filePath, List<DynSearchFieldDto> searchKeys, List<String> groupKeys, List<String> testItemNames,
                                  YieldItemParamDto yieldItemParamDto, String exportFilePath, ProcessHelper processHelper, ExportThreadManager exportThreadManager) {
        try {
            //assemble the param of drawing excel
            List<YieldDto> yieldDtoList = getYield(filePath, searchKeys, groupKeys, testItemNames, null);
            processHelper.postProcess(45, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DATA));

            YieldParamDto yieldParamDto = new YieldParamDto();
            yieldParamDto.setFilePath(filePath);
            yieldParamDto.setGroupKeys(groupKeys);
            yieldParamDto.setSearchKeys(searchKeys);
            yieldParamDto.setTestItemNames(testItemNames);
            Double usl = null;
            Double lsl = null;
            if (yieldItemParamDto != null) {
                usl = yieldItemParamDto.getUsl();
                lsl = yieldItemParamDto.getLsl();
            }
            yieldParamDto.setUsl(usl);
            yieldParamDto.setLsl(lsl);

            if (!this.isNullExportThread(exportThreadManager)) {
                return false;
            }

            //fill data to cell list
            YieldExportWorker factory = new YieldExportWorker();
            factory.buildYieldSunmmy(yieldParamDto, yieldDtoList);
            processHelper.postProcess(70, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DONE));

            if (!this.isNullExportThread(exportThreadManager)) {
                return false;
            }

            //draw excel
            YieldExcelBuilder yieldExcelBuilder = new YieldExcelBuilder(null, null);
            yieldExcelBuilder.drawExcel(exportFilePath, factory);
            logger.debug("Export yield only result to filepath:{}", exportFilePath);
        } catch (ApplicationException e) {
            logger.error("export grr result and data failed!", e);
            throw new ApplicationException(e.getCode(), e.getMessage());
        }
        return true;
    }

    /*
      * Multithreaded export Summary and Detail of Yield
      * @param filePath          source file path
      * @param searchKeys        search condition
      * @param groupKeys         grouping
      * @param testItemNames     name of test item
      * @param yieldItemParamDto
      * @param exportFilePath    file exporting path
      * @param processHelper     progress bar
      * @param exportThreadManager  class of Multithreaded export
      * @return                  it will return true when successfully export excel
      */
    private boolean exportSummaryAndDetail(List<String> filePath, List<DynSearchFieldDto> searchKeys, List<String> groupKeys, List<String> testItemNames,
                                           YieldItemParamDto yieldItemParamDto, String exportFilePath, String fileName, ProcessHelper processHelper, ExportThreadManager exportThreadManager) {
        try {
            processHelper.postProcess(25, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_START_EXPORT));
            YieldExportDto yieldExportDto = getSummaryAndDetail(filePath, searchKeys, groupKeys, testItemNames, yieldItemParamDto);
            processHelper.postProcess(55, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DATA));

            if (!this.isNullExportThread(exportThreadManager)) {
                return false;
            }

            //assemble the param of drawing excel
            List<YieldDto> yieldDtoList = Lists.newArrayList();
            if (!yieldExportDto.equals(null)) {
                yieldDtoList = yieldExportDto.getYieldDtos();
            }
            YieldParamDto yieldParamDto = new YieldParamDto();
            yieldParamDto.setFilePath(filePath);
            yieldParamDto.setGroupKeys(groupKeys);
            yieldParamDto.setSearchKeys(searchKeys);
            if (testItemNames != null && !testItemNames.isEmpty()) {
                yieldParamDto.setTestItemNames(testItemNames);
                Double usl = null;
                Double lsl = null;
                if (yieldItemParamDto != null) {
                    usl = yieldItemParamDto.getUsl();
                    lsl = yieldItemParamDto.getLsl();
                }
                yieldParamDto.setUsl(usl);
                yieldParamDto.setLsl(lsl);
            }

            final int count = 255;
            int startIndex = 0;
            int endIndex = 0;

            for (int i = 0; i < yieldDtoList.size(); i += count) {
                if (!this.isNullExportThread(exportThreadManager)) {
                    return false;
                }
                startIndex = i;
                endIndex = i + count;
                if (endIndex > yieldDtoList.size()) {
                    endIndex = yieldDtoList.size();
                }
                List<YieldDto> partyieldDtoList = yieldDtoList.subList(startIndex, endIndex);
                List<List<YieldHistoryDto>> partyieldYieldHistoryDtoList = getPart(partyieldDtoList, yieldExportDto.getYieldHistoryDtoList());

                if (partyieldYieldHistoryDtoList.size() == 1 && partyieldYieldHistoryDtoList.get(0) == null) {
                    SystemConfigDto systemConfigDto = systemService.findSystemConfig();
                    if (systemConfigDto == null || StringUtils.isBlank(systemConfigDto.getPrimaryKey())
                            || StringUtils.isBlank(systemConfigDto.getYieldValidateKey()) || StringUtils.isBlank(systemConfigDto.getYieldValidatePassValue())) {
                        logger.error("primaryKey or validate pass key is empty.");
                        throw new ApplicationException(ExceptionMessages.ERR_13004, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_YIELD_EXPORT_PARAMETER_INVALID
                        ));
                    }
                    String scriptPath = this.getScriptPath();
                    Set<String> seachKeyResult = this.getSearchAndAttributeKeys(filePath, scriptPath, systemConfigDto);
                    if (testItemNames != null && testItemNames.size() != 0) {
                        seachKeyResult.add(testItemNames.get(0));
                    }
                    List<YieldHistoryDto> yieldHistoryDtos = Lists.newArrayList();
                    YieldHistoryDto yieldHistoryDto = new YieldHistoryDto();
                    yieldHistoryDto.setColumnNames((String[]) seachKeyResult.toArray(new String[seachKeyResult.size()]));
                    for (int j = 0; j < 6; j++) {
                        yieldHistoryDtos.add(j, yieldHistoryDto);
                    }
                    partyieldYieldHistoryDtoList.clear();
                    partyieldYieldHistoryDtoList.add(0, yieldHistoryDtos);
                }

                //fill data to cell list
                YieldExportWorker factory = new YieldExportWorker();
                factory.buildYieldSunmmyAndDetail(yieldParamDto, partyieldDtoList, partyieldYieldHistoryDtoList);
                processHelper.postProcess(70, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DONE));

                if (!this.isNullExportThread(exportThreadManager)) {
                    return false;
                }

                //draw excel
                YieldExcelBuilder yieldExcelBuilder = new YieldExcelBuilder(null, null);
                yieldExcelBuilder.drawExcel(exportFilePath, factory);
                processHelper.postProcess(90, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CREATE_FILE) + fileName + "_" + ((i / count) + 1));
                logger.debug("Export yield only result to export file Path:{}", exportFilePath);
            }
            yieldParamDto = null;
            yieldDtoList = null;

        } catch (ApplicationException e) {
            logger.error("export grr result and data failed!", e);
            throw new ApplicationException(e.getCode(), e.getMessage());
        }
        return true;
    }

    private List<List<YieldHistoryDto>> getPart(List<YieldDto> partyieldDtoList, Map<String, List<YieldHistoryDto>> yieldHistory) {
        List<List<YieldHistoryDto>> partList = new ArrayList<>();
        for (int i = 0; i < partyieldDtoList.size(); i++) {
            try {
                List<YieldHistoryDto> yieldHistoryDtos = yieldHistory.get(partyieldDtoList.get(i).getGroupKey());
                partList.add(yieldHistoryDtos);
            } catch (IndexOutOfBoundsException e) {
                break;
            }
        }
        return partList;
    }

    private List<String> getAllColNames(List<List<String>> colNamesObject) {
        List<String> allColNames = new ArrayList<>();
        colNamesObject.forEach(allColNames::addAll);

        return allColNames;
    }

    private String checkSearchKeys(PythonUtil util, List<String> filePaths, List<String> configKeys) {
        String errorResult = "";
        for (String filePath : filePaths) {
            List<String> filePathsTemp = Lists.newArrayList();
            filePathsTemp.add(filePath);
            List<String> allColNames = getAllColNames(jsonMapper.fromJson(util.getColumnKeys(filePathsTemp, false).toString(), List.class));
            //check if searchKeys which come from files configuration all exist in files.
            //after check,(if configuration is right )the param searchKeys will keep null,otherwise it will keep the false searchKey names.
            String error = "";
            for (String column : configKeys) {
                if (!allColNames.contains(column)) {
                    error += column + System.getProperty("line.separator");
                }
            }
            if (StringUtils.isNotBlank(error)) {
                String name = filePath.substring(filePath.lastIndexOf(File.separator) + 1, filePath.length());
                String errorMessage = name + System.getProperty("line.separator") + error;
                errorResult += errorMessage + System.getProperty("line.separator");
            }
            allColNames = null;
        }

        return errorResult;
    }

    private boolean isNullExportThread(ExportThreadManager exportThreadManager) {
        if (exportThreadManager != null) {
            Thread exportThread = exportThreadManager.getYieldExportThreads().get(Thread.currentThread().getName());
            if (exportThread == null) {
                return false;
            }
        }
        return true;
    }

}
